# -*- coding: utf-8 -*-
"""
Houzz数据爬虫主程序
支持列表页和详情页数据获取，使用Redis进行任务管理
移除MySQL依赖，所有数据存储到Redis
"""

import sys
import time
import signal
import threading
from typing import List
from modules.common import RedisManager, ProxyManager, ExcelExporter, get_logger
from modules.common.logger_manager import log_task_start, log_task_success, log_task_failure
from modules.houzz import ListPageScraper, DetailPageScraper
from config import get_spider_config

# Houzz 爬虫配置
spider_config = get_spider_config('houzz')
SPIDER_NAME = spider_config.SPIDER_NAME
CATEGORIES = spider_config.CATEGORIES

logger = get_logger('main', spider_name=SPIDER_NAME)


class HouzzScraperMain:
    """Houzz爬虫主程序"""
    
    def __init__(self, spider_name: str = SPIDER_NAME):
        """初始化主程序"""
        self.spider_name = spider_name
        self.redis_manager = None
        self.proxy_manager = None
        self.list_scraper = None
        self.detail_scraper = None
        self.excel_exporter = None
        self.running = False
        self.threads = []
        
        # 注册信号处理器
        signal.signal(signal.SIGINT, self._signal_handler)
        signal.signal(signal.SIGTERM, self._signal_handler)
    
    def _signal_handler(self, signum, frame):
        """信号处理器"""
        logger.info(f"收到信号 {signum}，正在停止程序...")
        self.stop()
        sys.exit(0)
    
    def initialize(self):
        """初始化所有组件"""
        try:
            logger.info(f"🚀 初始化{self.spider_name}爬虫系统")
            
            # 初始化Redis管理器
            logger.info("初始化Redis管理器...")
            self.redis_manager = RedisManager(spider_name=self.spider_name)
            
            # 初始化代理管理器
            logger.info("初始化代理管理器...")
            self.proxy_manager = ProxyManager()
            
            # 初始化爬虫
            logger.info("初始化爬虫模块...")
            self.list_scraper = ListPageScraper(self.redis_manager, self.proxy_manager)
            self.detail_scraper = DetailPageScraper(self.redis_manager, self.proxy_manager)
            
            # 初始化Excel导出器
            logger.info("初始化Excel导出器...")
            self.excel_exporter = ExcelExporter(self.redis_manager)
            
            logger.info("✅ 系统初始化完成")
            return True
            
        except Exception as e:
            logger.error(f"❌ 系统初始化失败: {e}")
            return False
    
    def create_initial_tasks(self):
        """创建初始任务"""
        try:
            logger.info("创建初始列表页任务...")
            self.list_scraper.create_initial_tasks(CATEGORIES)
            
            # 显示任务统计
            stats = self.redis_manager.get_stats()
            logger.info(f"任务创建完成，当前统计: {stats}")
            
        except Exception as e:
            logger.error(f"创建初始任务失败: {e}")
            raise
    
    def start_list_scraper(self):
        """启动列表页爬虫"""
        def run_list_scraper():
            try:
                self.list_scraper.run_continuous()
            except Exception as e:
                logger.error(f"列表页爬虫运行错误: {e}")
        
        thread = threading.Thread(target=run_list_scraper, name="ListScraper")
        thread.daemon = True
        thread.start()
        self.threads.append(thread)
        logger.info("✅ 列表页爬虫已启动")
    
    def start_detail_scraper(self):
        """启动详情页爬虫"""
        def run_detail_scraper():
            try:
                self.detail_scraper.run_continuous()
            except Exception as e:
                logger.error(f"详情页爬虫运行错误: {e}")
        
        thread = threading.Thread(target=run_detail_scraper, name="DetailScraper")
        thread.daemon = True
        thread.start()
        self.threads.append(thread)
        logger.info("✅ 详情页爬虫已启动")
    
    def start_monitor(self):
        """启动监控线程"""
        def run_monitor():
            while self.running:
                try:
                    time.sleep(60)  # 每分钟检查一次
                    
                    if not self.running:
                        break
                    
                    # 获取统计信息
                    stats = self.redis_manager.get_stats()
                    logger.info("📊 系统统计信息:")
                    for key, value in stats.items():
                        logger.info(f"  - {key}: {value}")
                    
                    # 获取列表页任务详细统计
                    list_stats = self.redis_manager.get_list_task_stats()
                    if list_stats:
                        logger.info(f"  - 列表页任务总数: {list_stats.get('total_tasks', 0)}")
                        category_stats = list_stats.get('category_stats', {})
                        if category_stats:
                            logger.info("  - 各类目任务分布:")
                            for category, count in category_stats.items():
                                # 使用正则表达式提取类目名称
                                import re
                                pattern = r'/professionals/([^/]+)(?:/|$)'
                                match = re.search(pattern, category)
                                if match:
                                    category_name = match.group(1)
                                else:
                                    # 备用方法
                                    category_name = category.split('/')[-1].split('~')[0] if '~' in category else category.split('/')[-1]
                                logger.info(f"    - {category_name}: {count} 个任务")
                    
                    # 重试失败任务
                    # if stats.get('list_failed_tasks', 0) > 0:
                    #     logger.info("发现列表页失败任务，开始重试...")
                    #     self.list_scraper.retry_failed_tasks()
                    
                    # if stats.get('detail_failed_tasks', 0) > 0:
                    #     logger.info("发现详情页失败任务，开始重试...")
                    #     self.detail_scraper.retry_failed_tasks()
                    
                except Exception as e:
                    logger.error(f"监控线程错误: {e}")
        
        thread = threading.Thread(target=run_monitor, name="Monitor")
        thread.daemon = True
        thread.start()
        self.threads.append(thread)
        logger.info("✅ 监控线程已启动")
    
    def export_data(self):
        """导出数据到Excel"""
        try:
            logger.info("开始导出数据到Excel...")
            
            # 导出主数据
            if self.excel_exporter.export_to_excel():
                logger.info("✅ 主数据导出成功")
            else:
                logger.warning("❌ 主数据导出失败")
            
            # 导出失败任务
            if self.excel_exporter.export_failed_tasks():
                logger.info("✅ 失败任务导出成功")
            else:
                logger.warning("❌ 失败任务导出失败")
            
            # 导出汇总报告
            if self.excel_exporter.export_summary_report():
                logger.info("✅ 汇总报告导出成功")
            else:
                logger.warning("❌ 汇总报告导出失败")
                
        except Exception as e:
            logger.error(f"导出数据失败: {e}")
    
    def start(self):
        """启动爬虫系统"""
        try:
            # 初始化系统
            if not self.initialize():
                return False
            
            # 创建初始任务
            self.create_initial_tasks()
            
            # 设置运行状态
            self.running = True
            
            # 启动各个组件
            self.start_list_scraper()
            self.start_detail_scraper()
            self.start_monitor()
            
            logger.info("🎉 爬虫系统启动完成，开始运行...")
            return True
            
        except Exception as e:
            logger.error(f"启动爬虫系统失败: {e}")
            return False
    
    def stop(self):
        """停止爬虫系统"""
        logger.info("🛑 正在停止爬虫系统...")
        self.running = False
        
        # 等待所有线程结束
        for thread in self.threads:
            if thread.is_alive():
                thread.join(timeout=5)
        
        # 导出数据
        # logger.info("导出最终数据...")
        # self.export_data()
        
        logger.info("✅ 爬虫系统已停止")
    
    def run(self):
        """运行主程序"""
        try:
            if not self.start():
                logger.error("启动失败，程序退出")
                return False
            
            # 主循环
            while self.running:
                try:
                    time.sleep(1)
                except KeyboardInterrupt:
                    logger.info("收到中断信号")
                    break
            
            self.stop()
            return True
            
        except Exception as e:
            logger.error(f"主程序运行错误: {e}")
            return False


def main():
    """主函数"""
    import argparse
    
    # 解析命令行参数
    parser = argparse.ArgumentParser(description=f'{SPIDER_NAME}爬虫主程序')
    parser.add_argument('--spider-name', type=str, default=SPIDER_NAME,
                       help=f'爬虫名称，默认为{SPIDER_NAME}')
    args = parser.parse_args()
    
    try:
        # 创建并运行爬虫
        scraper = HouzzScraperMain(spider_name=args.spider_name)
        success = scraper.run()
        
        if success:
            logger.info("程序正常退出")
            sys.exit(0)
        else:
            logger.error("程序异常退出")
            sys.exit(1)
            
    except Exception as e:
        logger.error(f"程序启动失败: {e}")
        sys.exit(1)


if __name__ == "__main__":
    main()